#!/usr/local/python3/bin/python3
# -*- coding: UTF-8 -*-
import json
from elasticsearch import Elasticsearch
from elasticsearch import helpers
import datetime

es_index = 'patentabstract'
es_type = 'patent'
json_path = "/opt/patentabstractscript/patent_abstract_json_path.txt"


# 日期格式化函数
def convert_date(date):
    date_str = date.replace('T', ' ')[0:19]
    return datetime.datetime.strptime(date_str, '%Y-%m-%d %H:%M:%S')


# 批量保存到es
def bulk_insert(array, s, e):
    insert_flag = False
    try:
        helpers.bulk(es, array)
        insert_flag = True
    except BaseException:
        try:
            helpers.bulk(es, array)
            insert_flag = True
        except BaseException:
            try:
                helpers.bulk(es, array)
                insert_flag = True
            except BaseException:
                insert_flag = False

    if insert_flag:
        print("Save success:%d to %d" % (s, e))
    else:
        print("Save failed:%d to %d" % (s, e))


bulk_insert_num = 0
# 读取json文件路径
json_path_reader = open(json_path, encoding="utf-8")
file_path = json_path_reader.readline().strip()
if file_path == '':
    print('Please fill in the json file path')
else:
    print('Start reading json file : ', file_path)
    file = open(file_path.strip(), encoding="utf-8")
    line = file.readline()

    target_array = []

    # es连接获取
    es = Elasticsearch(hosts=[{'host': "localhost", 'port': "9200"}], timeout=30)
    while line:
        data = json.loads(line)
        # 获取json文件的数据
        proxyOrg = ''
        if 'proxyOrg' in data:
            if data['proxyOrg'] is not None:
                proxyOrg = data['proxyOrg']

        source = {}
        if proxyOrg != '':
            source = {
                "patentAbstract": data['patentAbstract'],
                "pid": data['pid'],
                "publishDate": convert_date(data['publishDate']['$date']),
                "appDate": convert_date(data['appDate']['$date']),
                "patentType": data['patentType'],
                "proxyOrg": proxyOrg
            }
        else:
            source = {
                "patentAbstract": data['patentAbstract'],
                "pid": data['pid'],
                "publishDate": convert_date(data['publishDate']['$date']),
                "appDate": convert_date(data['appDate']['$date']),
                "patentType": data['patentType']
            }
        # es插入数据格式构建
        target = {"_index": es_index, "_type": es_type, "_id": data['_id']['$oid'], "_source": source}
        target_array.append(target)

        # 数组内部数量达到1000则往es插入一次
        if len(target_array) == 1000:
            start = bulk_insert_num
            bulk_insert_num = bulk_insert_num + 1000
            end = bulk_insert_num
            bulk_insert(target_array, start, end)
            target_array.clear()

        line = file.readline()

    # 最后一次不足1000的内容插入es
    if len(target_array) > 0:
        start = bulk_insert_num
        bulk_insert_num = bulk_insert_num + len(target_array)
        end = bulk_insert_num
        bulk_insert(target_array, start, end)
        target_array.clear()
    file.close()
    print('End reading')
json_path_reader.close()
